import re

import pandas as pd
import seaborn as sns
sns.set()
import plotly.express as px
import plotly.graph_objects as go
df = pd.read_csv("indicators.csv.gz")
df.head()
setup FE algo indicator nobj problem nvar seed value
0 default 2500 cma rpd 2 DTLZ2 30 1 0.030942
1 default 2500 cma rpd 2 DTLZ2 30 2 0.024093
2 default 2500 cma rpd 2 DTLZ2 30 3 0.032611
3 default 2500 cma rpd 2 DTLZ2 30 4 0.035231
4 default 2500 cma rpd 2 DTLZ2 30 5 0.035426
rpd_outliers = "indicator == 'rpd' and 0 <= value <= 0.4"
eps_outliers = "indicator == 'eps' and 0 <= value <= 4"
igd_outliers = "indicator == 'igd' and 0 <= value <= 10"

Figure 1

df_no_moga = df.query("algo != 'moga'")
df_no_moga_nor_outliers = df_no_moga.query(f"{rpd_outliers} or {eps_outliers} or {igd_outliers}")
df_tuned_default = df_no_moga_nor_outliers.pivot_table(
    index=["FE", "algo", "indicator", "nobj", "problem", "nvar"], 
    columns=["setup"], 
    values=["value"]
)
df_tuned_default = df_tuned_default.droplevel(0, axis=1).reset_index()
df_tuned_default
setup FE algo indicator nobj problem nvar default tuned
0 2500 cma eps 2 DTLZ2 30 1.279888 1.533545
1 2500 cma eps 2 DTLZ2 40 1.983566 2.099601
2 2500 cma eps 2 DTLZ2 50 2.614604 2.657629
3 2500 cma eps 2 DTLZ4 30 1.139376 2.310213
4 2500 cma eps 2 DTLZ4 40 1.992210 2.849620
... ... ... ... ... ... ... ... ...
11524 40000 spea rpd 10 WFG8 41 0.167205 0.016128
11525 40000 spea rpd 10 WFG8 51 0.168898 0.018795
11526 40000 spea rpd 10 WFG9 31 0.130161 0.159438
11527 40000 spea rpd 10 WFG9 41 0.153218 0.183199
11528 40000 spea rpd 10 WFG9 51 0.198191 0.204703

11529 rows × 8 columns

def add_line(fig, xmax, col):
    fig.add_scatter(
        x=[0, xmax], 
        y=[0, xmax], 
        mode="lines", 
        line=go.scatter.Line(color="red"),
        row=1, 
        col=col, 
        showlegend=False,
    )
fig = px.scatter( 
    df_tuned_default,
    x="tuned",
    y="default",
    facet_col="indicator",
    category_orders={"indicator": ["rpd", "eps", "igd"]},
    height=400,
)

for k in fig.layout:
    if re.search('xaxis[1-9]+', k): 
        fig.layout[k].update(matches=None)
for k in fig.layout: 
    if re.search('yaxis[1-9]+', k): 
        fig.layout[k].update(matches=None)

add_line(fig, 0.4, 1)
add_line(fig, 4, 2)
add_line(fig, 10, 3)

fig

Figure 2

df_anytime = pd.read_csv("anytime.csv.gz")
df_anytime["config"] = df_anytime["config"].fillna("default")
df_anytime
setup config algo indicator nobj problem nvar seed 0 100 ... 49100 49200 49300 49400 49500 49600 49700 49800 49900 50000
0 default default ibea rpd 3 DTLZ2 30 1 0.018914 0.018914 ... 0.000026 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025
1 default default ibea rpd 3 DTLZ2 30 2 0.014705 0.014705 ... 0.000026 0.000025 0.000025 0.000026 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025
2 default default ibea rpd 3 DTLZ2 30 3 0.012488 0.012488 ... 0.000026 0.000025 0.000026 0.000026 0.000026 0.000025 0.000025 0.000025 0.000025 0.000024
3 default default ibea rpd 3 DTLZ2 30 4 0.016872 0.016872 ... 0.000025 0.000024 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025 0.000025
4 default default ibea rpd 3 DTLZ2 30 5 0.017557 0.017557 ... 0.000026 0.000026 0.000026 0.000025 0.000025 0.000025 0.000026 0.000026 0.000025 0.000024
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
7345 tuned 10000 spea rpd 3 WFG9 50 21 0.319248 0.273683 ... 0.009358 0.009483 0.009145 0.009404 0.008750 0.009153 0.009152 0.009182 0.009248 0.009218
7346 tuned 10000 spea rpd 3 WFG9 50 22 0.287983 0.269812 ... 0.022561 0.021796 0.022061 0.022061 0.022061 0.022061 0.022061 0.022061 0.022061 0.022061
7347 tuned 10000 spea rpd 3 WFG9 50 23 0.338829 0.283009 ... 0.009563 0.010079 0.007908 0.007795 0.007770 0.007758 0.007855 0.009834 0.009410 0.009380
7348 tuned 10000 spea rpd 3 WFG9 50 24 0.316427 0.300587 ... 0.009780 0.009780 0.009780 0.009780 0.009780 0.009780 0.009780 0.009780 0.009780 0.009780
7349 tuned 10000 spea rpd 3 WFG9 50 25 0.317221 0.302947 ... 0.014507 0.014507 0.014507 0.014507 0.014507 0.014507 0.014507 0.014507 0.014507 0.014507

7350 rows × 509 columns

df_anytime_mean = df_anytime.groupby(["setup", "config", "algo", "indicator", "nobj", "problem", "nvar"]).mean()
df_anytime_mean = df_anytime_mean.drop(columns=["seed"])

ts_anytime = df_anytime_mean.stack().reset_index(
    ["setup", "config", "algo", "indicator", "nobj", "problem", "nvar"], 
    name="value"
)
ts_anytime.index = ts_anytime.index.astype("int")
ts_anytime
setup config algo indicator nobj problem nvar value
0 default default ibea rpd 3 DTLZ2 30 0.016579
100 default default ibea rpd 3 DTLZ2 30 0.016579
200 default default ibea rpd 3 DTLZ2 30 0.016579
300 default default ibea rpd 3 DTLZ2 30 0.010874
400 default default ibea rpd 3 DTLZ2 30 0.008861
... ... ... ... ... ... ... ... ...
49600 tuned 40000 ibea rpd 3 WFG9 50 0.003880
49700 tuned 40000 ibea rpd 3 WFG9 50 0.003826
49800 tuned 40000 ibea rpd 3 WFG9 50 0.003777
49900 tuned 40000 ibea rpd 3 WFG9 50 0.003810
50000 tuned 40000 ibea rpd 3 WFG9 50 0.003835

147294 rows × 8 columns

ts_ibea_WFG8_3_30 = ts_anytime.query("algo == 'ibea' and problem == 'WFG8' and nobj == 3 and nvar == 30")\
                              .sort_index()
fig2 = px.line(
    ts_ibea_WFG8_3_30,
    y="value",
    x=ts_ibea_WFG8_3_30.index,
    color="config",
    line_dash="config",
)
fig2.show()
ts_ibea = ts_anytime.query("algo == 'ibea'")
fig2_full = px.line(
    ts_ibea,
    y="value",
    x=ts_ibea.index,
    color="config",
    line_dash="config",
    animation_frame="nvar",
    facet_col="problem",
    facet_col_wrap=3,
    height=1500,
)
        
fig2_full.show()
df_ibea_snapshots = ts_ibea.loc[[2500,10000,40000]].reset_index().rename(columns={"index": "FE"})
rs_ibea_snapshots = df_ibea_snapshots.drop(columns=["setup", "algo", "indicator"])\
                                     .pivot_table(index=["problem", "nobj", "nvar", "FE"], columns=["config"])\
                                     .rank(axis=1).groupby("FE").sum()

for FE in [2500, 10000, 40000]:
    rs_ibea_diff = (rs_ibea_snapshots.loc[FE] - rs_ibea_snapshots.loc[FE].min())
    display(rs_ibea_diff.sort_values().to_frame().T)
value
config 2500.0 10000.0 40000.0 default
2500 0.0 30.0 76.0 90.0
value
config 10000.0 40000.0 2500.0 default
10000 0.0 19.0 46.0 59.0
value
config 40000.0 10000.0 default 2500.0
40000 0.0 30.0 33.0 93.0

Figure 4

ts_10k_WFG3_3_50 = ts_anytime.query("config == 10000 and problem == 'WFG3' and nobj == 3 and nvar == 50")\
                             .sort_index()
fig4 = px.line(
    ts_10k_WFG3_3_50,
    y="value",
    x=ts_10k_WFG3_3_50.index,
    color="algo",
    line_dash="algo",
)
fig4.show()
ts_10k = ts_anytime.query("config == 10000")
fig4_full = px.line(
    ts_10k,
    y="value",
    x=ts_10k.index,
    color="algo",
    line_dash="algo",
    animation_frame="nvar",
    facet_col="problem",
    facet_col_wrap=3,
    height=1500,
)

for k in fig4_full.layout: 
    if re.search('yaxis[1-9]+', k): 
        fig4_full.layout[k].update(matches=None)
        
fig4_full.show()

Figure 5

algo_fig5 = ["ibea", "sms"]
ind_fig5 = ["rpd", "igd"]
scenario_fig5 = "problem == 'WFG8' and nobj == 2 and nvar == 30 and FE == 10000"
df_fig5 = df.query(f"algo in {algo_fig5} and indicator in {ind_fig5} and {scenario_fig5} and setup == 'tuned'")

fig5 = px.box(
    df_fig5,
    y="algo",
    x="value",
    color="algo",
    facet_col="indicator",
    height=300,
    width=600,
)
fig5.show()
df_tuned_no_outliers = df.query(f"setup == 'tuned' and {rpd_outliers} or {eps_outliers} or {igd_outliers}")
fig5_full = px.box(
    df_tuned_no_outliers,
    x="nvar",
    y="value",
    color="algo",
    facet_col="nobj",
    facet_row="problem",
    animation_frame="indicator",
    height=4000,
    category_orders={"indicator": ["rpd", "eps", "igd"]}
#     width=600,
)

# for k in fig5_full.layout: 
#     if re.search('yaxis[1-9]+', k): 
#         fig5_full.layout[k].update(matches=None)
        
fig5_full.show()

Table 6

def rank_sum(df, columns=["algo"]):
    df_wide = df.pivot_table(
        index=["indicator", "problem", "nvar", "seed"],
        columns=columns, 
        values=["value"]
    )
    
    return df_wide.rank(axis=1).groupby("indicator").sum()
df_rs = df.groupby(["setup","FE", "nobj"]).apply(rank_sum).droplevel(0, axis=1)
long_rs_tuned = df_rs.query("setup == 'tuned'").stack().reset_index(name="value")
long_rs_tuned.pivot_table(
    index=["indicator", "nobj", "algo"],
    columns=["FE"], 
    values=["value"]
).droplevel(0, axis=1).corr()
FE 2500 10000 40000
FE
2500 1.000000 0.862971 0.813624
10000 0.862971 1.000000 0.940250
40000 0.813624 0.940250 1.000000
long_rs_default = df_rs.query("setup == 'default'").stack().reset_index(name="value")
long_rs_default.pivot_table(
    index=["indicator", "nobj", "algo"],
    columns=["FE"], 
    values=["value"]
).droplevel(0, axis=1).corr()
FE 2500 10000 40000
FE
2500 1.000000 0.771288 0.676950
10000 0.771288 1.000000 0.959275
40000 0.676950 0.959275 1.000000

Table 7

long_rs_tuned.groupby("nobj").apply(lambda x : x.pivot_table(
    index=["FE", "algo"],
    columns=["indicator"], 
    values=["value"]
).droplevel(0, axis=1).corr())
indicator eps igd rpd
nobj indicator
2 eps 1.000000 0.952154 0.971495
igd 0.952154 1.000000 0.888355
rpd 0.971495 0.888355 1.000000
3 eps 1.000000 0.875681 0.938570
igd 0.875681 1.000000 0.857850
rpd 0.938570 0.857850 1.000000
5 eps 1.000000 0.902089 0.949822
igd 0.902089 1.000000 0.852685
rpd 0.949822 0.852685 1.000000
10 eps 1.000000 0.353728 0.503038
igd 0.353728 1.000000 0.776752
rpd 0.503038 0.776752 1.000000

Table 8

for FE in [2500, 10000, 40000]:
    for nobj in [2,3,5,10]:
        for indicator in ["rpd", "eps", "igd"]:
            idx = ("tuned", FE, nobj, indicator)
            rs_diff = (df_rs.loc[idx] - df_rs.loc[idx].min())
            display(rs_diff.sort_values().to_frame().T)
algo sms ibea nsga spea moead hype nsga3 cma moga
tuned 2500 2 rpd 0.0 11.0 456.0 971.0 1972.0 2124.5 2396.5 5139.5 5622.5
algo sms ibea nsga spea hype nsga3 moead cma moga
tuned 2500 2 eps 0.0 159.0 1806.0 2048.0 2071.5 2446.5 3976.0 5766.5 6017.5
algo sms ibea nsga3 spea hype nsga cma moead moga
tuned 2500 2 igd 0.0 490.0 2979.5 3025.0 3152.5 3172.0 5494.5 6026.0 6386.5
algo sms ibea hype moead spea nsga nsga3 cma moga
tuned 2500 3 rpd 0.0 743.0 1516.0 2421.5 3464.0 3950.5 4068.5 4928.0 6727.5
algo sms ibea moead hype nsga nsga3 spea cma moga
tuned 2500 3 eps 0.0 53.0 2456.5 3102.0 3420.5 3580.5 4421.0 5495.0 6352.5
algo ibea moead sms nsga nsga3 spea hype cma moga
tuned 2500 3 igd 0.0 650.5 711.0 1434.5 2710.5 3592.0 3944.0 4883.0 5420.5
algo sms ibea moead spea nsga cma nsga3 hype moga
tuned 2500 5 rpd 0.0 1402.0 1603.5 3375.5 4245.5 4274.0 4731.5 4765.0 7463.0
algo sms ibea moead nsga3 nsga spea hype cma moga
tuned 2500 5 eps 0.0 745.0 1221.5 2898.5 3766.5 3785.5 4108.0 4365.0 5588.0
algo sms nsga3 ibea moead hype spea nsga cma moga
tuned 2500 5 igd 0.0 57.5 242.0 610.5 1034.0 1052.5 1931.5 3214.0 4503.0
algo sms ibea cma nsga nsga3 spea hype moead moga
tuned 2500 10 rpd 0.0 827.0 1919.5 2965.0 3543.0 4257.5 5525.0 6218.0 7505.0
algo moead ibea sms nsga cma nsga3 spea hype moga
tuned 2500 10 eps 0.0 370.0 2092.0 2471.0 2791.5 3315.0 3498.5 4878.0 6297.0
algo ibea nsga3 spea cma sms nsga hype moead moga
tuned 2500 10 igd 0.0 847.5 1272.5 1289.5 1915.0 2228.0 3315.0 4441.5 5562.0
algo ibea sms spea nsga moead hype cma nsga3 moga
tuned 10000 2 rpd 0.0 113.5 1149.5 1846.5 2819.5 3593.0 3731.5 4017.5 6682.5
algo sms ibea spea nsga moead hype cma nsga3 moga
tuned 10000 2 eps 0.0 425.0 894.0 1993.0 3091.5 3401.5 3749.0 4231.0 6740.0
algo sms spea ibea hype nsga moead nsga3 cma moga
tuned 10000 2 igd 0.0 711.0 1387.5 2382.5 2774.0 4237.0 5068.0 5240.0 7297.0
algo sms ibea moead hype spea cma nsga nsga3 moga
tuned 10000 3 rpd 0.0 556.0 1805.0 2290.0 2302.0 3616.0 4378.0 4627.5 7029.5
algo sms ibea spea cma hype moead nsga3 nsga moga
tuned 10000 3 eps 0.0 494.5 2516.0 2968.0 3132.0 3552.0 4253.0 4885.0 7323.5
algo ibea sms spea moead hype nsga cma nsga3 moga
tuned 10000 3 igd 0.0 654.0 1041.0 1622.0 2960.0 3640.0 4240.0 4264.5 6886.5
algo sms moead ibea spea cma nsga3 nsga hype moga
tuned 10000 5 rpd 0.0 1471.0 1535.0 3295.5 3374.0 3897.5 4130.5 5811.5 7562.0
algo sms ibea moead cma nsga nsga3 spea hype moga
tuned 10000 5 eps 0.0 1713.0 2569.0 2588.0 4086.5 4124.5 4701.5 5907.5 7664.0
algo sms ibea moead nsga cma spea hype nsga3 moga
tuned 10000 5 igd 0.0 1898.0 2119.0 2329.0 2515.5 3579.5 5040.5 5225.5 7398.0
algo ibea sms cma nsga3 spea nsga hype moead moga
tuned 10000 10 rpd 0.0 222.0 1116.0 2326.0 2532.0 3241.0 4846.0 5114.0 6919.0
algo moead ibea sms cma nsga3 nsga spea hype moga
tuned 10000 10 eps 0.0 1258.0 2794.0 3250.0 3347.0 4045.0 4562.0 5214.0 6922.0
algo nsga3 ibea spea nsga sms cma hype moead moga
tuned 10000 10 igd 0.0 36.0 646.0 1776.0 1828.0 1987.0 2557.0 4424.0 5151.0
algo sms ibea spea nsga moead nsga3 cma hype moga
tuned 40000 2 rpd 0.0 453.5 791.5 1795.5 2269.0 3492.0 3818.0 3888.5 6788.5
algo sms spea ibea nsga moead nsga3 cma hype moga
tuned 40000 2 eps 0.0 918.5 1197.5 2613.5 3053.5 3208.5 3393.5 4295.0 7060.0
algo sms spea ibea moead nsga3 nsga hype cma moga
tuned 40000 2 igd 0.0 731.5 2404.5 2833.0 3039.0 3409.5 4252.5 4446.0 7364.5
algo sms ibea moead spea hype cma nsga nsga3 moga
tuned 40000 3 rpd 0.0 715.0 1575.0 2715.5 3510.0 3604.0 4269.0 4276.5 7163.0
algo sms ibea cma spea moead nsga nsga3 hype moga
tuned 40000 3 eps 0.0 994.0 2614.0 2708.5 3137.0 4450.0 4774.5 5284.0 7610.0
algo moead sms spea ibea nsga cma nsga3 hype moga
tuned 40000 3 igd 0.0 269.0 763.5 1668.0 2780.0 3118.0 3984.5 4815.0 6686.0
algo sms ibea moead cma spea nsga3 nsga hype moga
tuned 40000 5 rpd 0.0 1154.0 1761.0 2915.0 2995.0 3325.5 4188.5 5826.5 7363.5
algo sms ibea cma moead spea nsga nsga3 hype moga
tuned 40000 5 eps 0.0 173.5 1610.5 1650.0 3310.0 3988.5 4436.5 5777.5 7097.5
algo ibea moead sms spea cma nsga nsga3 hype moga
tuned 40000 5 igd 0.0 244.0 827.0 1720.0 2074.0 2737.5 5087.5 5189.5 6780.5
algo ibea sms spea cma nsga3 nsga hype moead moga
tuned 40000 10 rpd 0.0 897.0 1401.0 1878.0 2416.0 2576.0 4899.0 5077.0 7073.0
algo moead ibea nsga3 sms nsga spea cma hype moga
tuned 40000 10 eps 0.0 220.0 1836.0 2309.0 2986.0 3252.0 3705.0 4581.0 6518.0
algo ibea nsga3 spea nsga hype cma sms moead moga
tuned 40000 10 igd 0.0 928.0 942.0 2710.0 2937.0 3725.0 4129.0 5513.0 6584.0

Table 9

algo_tab9 = ["nsga", "nsga3"]
nobj_tab9 = [2,5]

df_nsga_tab9 = df.query(f"algo in {algo_tab9} and indicator == 'rpd' and nobj in {nobj_tab9} and FE == 10000")
rs_nsga_tab9 = df_nsga_tab9.groupby("nobj").apply(rank_sum, columns=["setup","algo"]).droplevel(0, axis=1)

for nobj in [2,5]:
    idx = (nobj, "rpd")
    rs_diff_tab9 = (rs_nsga_tab9.loc[idx] - rs_nsga_tab9.loc[idx].min())
    display(rs_diff_tab9.sort_values().to_frame().T)
setup tuned default tuned default
algo nsga nsga nsga3 nsga3
2 rpd 0.0 1270.5 1424.0 2257.5
setup tuned default
algo nsga nsga3 nsga3 nsga
5 rpd 0.0 11.0 901.0 1212.0

Figure 7

df_rpd_10k = df.query(f"indicator == 'rpd' and FE == 10000 and setup == 'tuned' and {rpd_outliers}")
df_rpd_10k["nobj"] = df_rpd_10k["nobj"].astype("category")
px.box(
    df_rpd_10k,
    x="nobj",
    color="nobj",
    y="value",
    facet_col="problem",
    facet_col_wrap=7,
    range_y=(0,0.4),
    points=False,
)
/home/a206/.local/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

Figure 8

problem_fig8 = ["WFG1", "WFG4"]
nvar_fig8 = [40, 41]
setup_fig8 = "FE == 40000 and indicator == 'igd' and setup == 'tuned'"
df_fig8 = df.query(f"problem in {problem_fig8} and nvar in {nvar_fig8} and {setup_fig8} and {igd_outliers}")

fig8 = px.box(
    df_fig8,
    x="value",
    y="algo",
    color="algo",
    facet_col="nobj",
    facet_row="problem",
    category_orders={"algo": ["cma", "hype", "ibea", "moead", "moga", "nsga", "nsga3", "sms", "spea"][::-1]},
)
fig8.show()